<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">
<html>
<head>
<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
<link rel="stylesheet" href="../../../../style.css" type="text/css" media="screen">
<link rel="stylesheet" href="../../../../print.css" type="text/css" media="print">
<meta content="Tokenizer,com.newgonzo.commons.parsing.Tokenizer,column,current,line,source,addTokenPattern,consume,createToken,getNextToken,lookAhead,nextToken,reset,skipToken" name="keywords">
<title>com.newgonzo.commons.parsing.Tokenizer</title>
</head>
<body>
<script type="text/javascript" language="javascript" src="../../../../asdoc.js"></script><script type="text/javascript" language="javascript" src="../../../../cookies.js"></script><script type="text/javascript" language="javascript">
<!--
				asdocTitle = 'Tokenizer - API Documentation';
				var baseRef = '../../../../';
				window.onload = configPage;
			--></script>
<table style="display:none" id="titleTable" cellspacing="0" cellpadding="0" class="titleTable">
<tr>
<td align="left" class="titleTableTitle">API Documentation</td><td align="right" class="titleTableTopNav"><a onclick="loadClassListFrame('../../../../all-classes.html')" href="../../../../package-summary.html">All&nbsp;Packages</a>&nbsp;|&nbsp;<a onclick="loadClassListFrame('../../../../all-classes.html')" href="../../../../class-summary.html">All&nbsp;Classes</a>&nbsp;|&nbsp;<a onclick="loadClassListFrame('../../../../index-list.html')" href="../../../../all-index-A.html">Index</a>&nbsp;|&nbsp;<a href="../../../../index.html?com/newgonzo/commons/parsing/Tokenizer.html&amp;com/newgonzo/commons/parsing/class-list.html" id="framesLink1">Frames</a><a onclick="parent.location=document.location" href="" style="display:none" id="noFramesLink1">No&nbsp;Frames</a></td><td rowspan="3" align="right" class="titleTableLogo"><img alt="Adobe Logo" title="Adobe Logo" class="logoImage" src="../../../../images/logo.jpg"></td>
</tr>
<tr class="titleTableRow2">
<td align="left" id="subTitle" class="titleTableSubTitle">Class&nbsp;Tokenizer</td><td align="right" id="subNav" class="titleTableSubNav"><a href="#propertySummary">Properties</a>&nbsp;| <a href="#methodSummary">Methods</a></td>
</tr>
<tr class="titleTableRow3">
<td colspan="2">&nbsp;</td>
</tr>
</table>
<script type="text/javascript" language="javascript">
<!--
if (!isEclipse() || window.name != ECLIPSE_FRAME_NAME) {titleBar_setSubTitle("Class Tokenizer"); titleBar_setSubNav(false,true,false,false,false,false,true,false,false,false,false,false,false,false);}
--></script>
<div class="MainContent">
<table cellspacing="0" cellpadding="0" class="classHeaderTable">
<tr>
<td class="classHeaderTableLabel">Package</td><td><a onclick="javascript:loadClassListFrame('class-list.html')" href="package-detail.html">com.newgonzo.commons.parsing</a></td>
</tr>
<tr>
<td class="classHeaderTableLabel">Class</td><td class="classSignature">public class Tokenizer</td>
</tr>
<tr>
<td class="classHeaderTableLabel">Subclasses</td><td><a href="../../cannonball/css/parser/CSSTokenizer.html">CSSTokenizer</a></td>
</tr>
</table>
<p></p>
<p></p>
<br>
<hr>
</div>
<a name="propertySummary"></a>
<div class="summarySection">
<div class="summaryTableTitle">Public Properties</div>
<table id="summaryTableProperty" class="summaryTable " cellpadding="3" cellspacing="0">
<tr>
<th>&nbsp;</th><th colspan="2">Property</th><th class="summaryTableOwnerCol">Defined&nbsp;by</th>
</tr>
<tr class="">
<td class="summaryTablePaddingCol">&nbsp;</td><td class="summaryTableInheritanceCol">&nbsp;</td><td class="summaryTableSignatureCol"><a class="signatureLink" href="#column">column</a> : int<div class="summaryTableDescription">[read-only]</div>
</td><td class="summaryTableOwnerCol">Tokenizer</td>
</tr>
<tr class="">
<td class="summaryTablePaddingCol">&nbsp;</td><td class="summaryTableInheritanceCol">&nbsp;</td><td class="summaryTableSignatureCol"><a class="signatureLink" href="#current">current</a> : <a href="../parsing/Token.html">Token</a>
<div class="summaryTableDescription">[read-only]</div>
</td><td class="summaryTableOwnerCol">Tokenizer</td>
</tr>
<tr class="">
<td class="summaryTablePaddingCol">&nbsp;</td><td class="summaryTableInheritanceCol">&nbsp;</td><td class="summaryTableSignatureCol"><a class="signatureLink" href="#line">line</a> : int<div class="summaryTableDescription">[read-only]</div>
</td><td class="summaryTableOwnerCol">Tokenizer</td>
</tr>
<tr class="">
<td class="summaryTablePaddingCol">&nbsp;</td><td class="summaryTableInheritanceCol">&nbsp;</td><td class="summaryTableSignatureCol"><a class="signatureLink" href="#source">source</a> : String<div class="summaryTableDescription"></div>
</td><td class="summaryTableOwnerCol">Tokenizer</td>
</tr>
</table>
</div>
<a name="methodSummary"></a>
<div class="summarySection">
<div class="summaryTableTitle">Public Methods</div>
<table id="summaryTableMethod" class="summaryTable " cellpadding="3" cellspacing="0">
<tr>
<th>&nbsp;</th><th colspan="2">Method</th><th class="summaryTableOwnerCol">Defined&nbsp;by</th>
</tr>
<tr class="">
<td class="summaryTablePaddingCol">&nbsp;</td><td class="summaryTableInheritanceCol">&nbsp;</td><td class="summaryTableSignatureCol">
<div class="summarySignature">
<a class="signatureLink" href="#Tokenizer()">Tokenizer</a>(source:String)</div>
<div class="summaryTableDescription"></div>
</td><td class="summaryTableOwnerCol">Tokenizer</td>
</tr>
<tr class="">
<td class="summaryTablePaddingCol">&nbsp;</td><td class="summaryTableInheritanceCol">&nbsp;</td><td class="summaryTableSignatureCol">
<div class="summarySignature">
<a class="signatureLink" href="#addTokenPattern()">addTokenPattern</a>(pattern:<a href="TokenPattern.html">TokenPattern</a>):void</div>
<div class="summaryTableDescription"></div>
</td><td class="summaryTableOwnerCol">Tokenizer</td>
</tr>
<tr class="">
<td class="summaryTablePaddingCol">&nbsp;</td><td class="summaryTableInheritanceCol">&nbsp;</td><td class="summaryTableSignatureCol">
<div class="summarySignature">
<a class="signatureLink" href="#lookAhead()">lookAhead</a>():<a href="../parsing/Token.html">Token</a>
</div>
<div class="summaryTableDescription"></div>
</td><td class="summaryTableOwnerCol">Tokenizer</td>
</tr>
<tr class="">
<td class="summaryTablePaddingCol">&nbsp;</td><td class="summaryTableInheritanceCol">&nbsp;</td><td class="summaryTableSignatureCol">
<div class="summarySignature">
<a class="signatureLink" href="#nextToken()">nextToken</a>():<a href="../parsing/Token.html">Token</a>
</div>
<div class="summaryTableDescription"></div>
</td><td class="summaryTableOwnerCol">Tokenizer</td>
</tr>
<tr class="">
<td class="summaryTablePaddingCol">&nbsp;</td><td class="summaryTableInheritanceCol">&nbsp;</td><td class="summaryTableSignatureCol">
<div class="summarySignature">
<a class="signatureLink" href="#reset()">reset</a>():void</div>
<div class="summaryTableDescription"></div>
</td><td class="summaryTableOwnerCol">Tokenizer</td>
</tr>
<tr class="">
<td class="summaryTablePaddingCol">&nbsp;</td><td class="summaryTableInheritanceCol">&nbsp;</td><td class="summaryTableSignatureCol">
<div class="summarySignature">
<a class="signatureLink" href="#skipToken()">skipToken</a>():void</div>
<div class="summaryTableDescription"></div>
</td><td class="summaryTableOwnerCol">Tokenizer</td>
</tr>
</table>
</div>
<a name="protectedMethodSummary"></a>
<div class="summarySection">
<div class="summaryTableTitle">Protected Methods</div>
<table id="summaryTableProtectedMethod" class="summaryTable " cellpadding="3" cellspacing="0">
<tr>
<th>&nbsp;</th><th colspan="2">Method</th><th class="summaryTableOwnerCol">Defined&nbsp;by</th>
</tr>
<tr class="">
<td class="summaryTablePaddingCol">&nbsp;</td><td class="summaryTableInheritanceCol">&nbsp;</td><td class="summaryTableSignatureCol">
<div class="summarySignature">
<a class="signatureLink" href="#consume()">consume</a>(chars:int = 1):void</div>
<div class="summaryTableDescription"> 
  Removes N characters from the front of the remaining source
  to tokenize.</div>
</td><td class="summaryTableOwnerCol">Tokenizer</td>
</tr>
<tr class="">
<td class="summaryTablePaddingCol">&nbsp;</td><td class="summaryTableInheritanceCol">&nbsp;</td><td class="summaryTableSignatureCol">
<div class="summarySignature">
<a class="signatureLink" href="#createToken()">createToken</a>(pattern:<a href="TokenPattern.html">TokenPattern</a>, source:String, sourceIndex:int, prevToken:<a href="Token.html">Token</a> = null):<a href="../parsing/Token.html">Token</a>
</div>
<div class="summaryTableDescription"></div>
</td><td class="summaryTableOwnerCol">Tokenizer</td>
</tr>
<tr class="">
<td class="summaryTablePaddingCol">&nbsp;</td><td class="summaryTableInheritanceCol">&nbsp;</td><td class="summaryTableSignatureCol">
<div class="summarySignature">
<a class="signatureLink" href="#getNextToken()">getNextToken</a>(consumeSource:Boolean = true):<a href="../parsing/Token.html">Token</a>
</div>
<div class="summaryTableDescription"></div>
</td><td class="summaryTableOwnerCol">Tokenizer</td>
</tr>
</table>
</div>
<script type="text/javascript" language="javascript">
<!--
showHideInherited();
--></script>
<div class="MainContent">
<a name="propertyDetail"></a>
<div class="detailSectionHeader">Property detail</div>
<a name="column"></a>
<table cellspacing="0" cellpadding="0" class="detailHeader">
<tr>
<td class="detailHeaderName">column</td><td class="detailHeaderType">property</td>
</tr>
</table>
<div class="detailBody">
<code>column:int</code>&nbsp;&nbsp;[read-only]<span class="label">Implementation</span>
<br>
<code>&nbsp;&nbsp;&nbsp;&nbsp;public function get column():int</code>
<br>
</div>
<a name="current"></a>
<table cellspacing="0" cellpadding="0" class="detailHeader">
<tr>
<td class="detailHeaderName">current</td><td class="detailHeaderType">property</td><td class="detailHeaderRule">&nbsp;</td>
</tr>
</table>
<div class="detailBody">
<code>current:<a href="../parsing/Token.html">Token</a></code>&nbsp;&nbsp;[read-only]<span class="label">Implementation</span>
<br>
<code>&nbsp;&nbsp;&nbsp;&nbsp;public function get current():<a href="../parsing/Token.html">Token</a></code>
<br>
</div>
<a name="line"></a>
<table cellspacing="0" cellpadding="0" class="detailHeader">
<tr>
<td class="detailHeaderName">line</td><td class="detailHeaderType">property</td><td class="detailHeaderRule">&nbsp;</td>
</tr>
</table>
<div class="detailBody">
<code>line:int</code>&nbsp;&nbsp;[read-only]<span class="label">Implementation</span>
<br>
<code>&nbsp;&nbsp;&nbsp;&nbsp;public function get line():int</code>
<br>
</div>
<a name="source"></a>
<table cellspacing="0" cellpadding="0" class="detailHeader">
<tr>
<td class="detailHeaderName">source</td><td class="detailHeaderType">property</td><td class="detailHeaderRule">&nbsp;</td>
</tr>
</table>
<div class="detailBody">
<code>source:String</code>&nbsp;&nbsp;[read-write]<span class="label">Implementation</span>
<br>
<code>&nbsp;&nbsp;&nbsp;&nbsp;public function get source():String</code>
<br>
<code>&nbsp;&nbsp;&nbsp;&nbsp;public function set source(value:String):void</code>
<br>
</div>
<a name="constructorDetail"></a>
<div class="detailSectionHeader">Constructor detail</div>
<a name="Tokenizer()"></a>
<table cellspacing="0" cellpadding="0" class="detailHeader">
<tr>
<td class="detailHeaderName">Tokenizer</td><td class="detailHeaderParens">()</td><td class="detailHeaderType">constructor</td>
</tr>
</table>
<div class="detailBody">
<code>public function Tokenizer(source:String)</code><span class="label">Parameters</span>
<table border="0" cellspacing="0" cellpadding="0">
<tr>
<td width="20px"></td><td><code><span class="label">source</span>:String</code></td>
</tr>
</table>
</div>
<a name="methodDetail"></a>
<div class="detailSectionHeader">Method detail</div>
<a name="addTokenPattern()"></a>
<table cellspacing="0" cellpadding="0" class="detailHeader">
<tr>
<td class="detailHeaderName">addTokenPattern</td><td class="detailHeaderParens">()</td><td class="detailHeaderType">method</td>
</tr>
</table>
<div class="detailBody">
<code>public function addTokenPattern(pattern:<a href="TokenPattern.html">TokenPattern</a>):void</code><span class="label">Parameters</span>
<table border="0" cellspacing="0" cellpadding="0">
<tr>
<td width="20px"></td><td><code><span class="label">pattern</span>:<a href="TokenPattern.html">TokenPattern</a></code></td>
</tr>
</table>
</div>
<a name="consume()"></a>
<table cellspacing="0" cellpadding="0" class="detailHeader">
<tr>
<td class="detailHeaderName">consume</td><td class="detailHeaderParens">()</td><td class="detailHeaderType">method</td><td class="detailHeaderRule">&nbsp;</td>
</tr>
</table>
<div class="detailBody">
<code>protected function consume(chars:int = 1):void</code><p> 
  Removes N characters from the front of the remaining source
  to tokenize.
  </p><span class="label">Parameters</span>
<table border="0" cellspacing="0" cellpadding="0">
<tr>
<td width="20px"></td><td><code><span class="label">chars</span>:int</code> (default = <code>1</code>)<code></code></td>
</tr>
</table>
</div>
<a name="createToken()"></a>
<table cellspacing="0" cellpadding="0" class="detailHeader">
<tr>
<td class="detailHeaderName">createToken</td><td class="detailHeaderParens">()</td><td class="detailHeaderType">method</td><td class="detailHeaderRule">&nbsp;</td>
</tr>
</table>
<div class="detailBody">
<code>protected function createToken(pattern:<a href="TokenPattern.html">TokenPattern</a>, source:String, sourceIndex:int, prevToken:<a href="Token.html">Token</a> = null):<a href="../parsing/Token.html">Token</a></code><span class="label">Parameters</span>
<table border="0" cellspacing="0" cellpadding="0">
<tr>
<td width="20px"></td><td><code><span class="label">pattern</span>:<a href="TokenPattern.html">TokenPattern</a></code></td>
</tr>
<tr>
<td class="paramSpacer">&nbsp;</td>
</tr>
<tr>
<td width="20px"></td><td><code><span class="label">source</span>:String</code></td>
</tr>
<tr>
<td class="paramSpacer">&nbsp;</td>
</tr>
<tr>
<td width="20px"></td><td><code><span class="label">sourceIndex</span>:int</code></td>
</tr>
<tr>
<td class="paramSpacer">&nbsp;</td>
</tr>
<tr>
<td width="20px"></td><td><code><span class="label">prevToken</span>:<a href="Token.html">Token</a></code> (default = <code>null</code>)<code></code></td>
</tr>
</table>
<p></p>
<span class="label">Returns</span>
<table border="0" cellspacing="0" cellpadding="0">
<tr>
<td width="20"></td><td><code><a href="../parsing/Token.html">Token</a></code></td>
</tr>
</table>
</div>
<a name="getNextToken()"></a>
<table cellspacing="0" cellpadding="0" class="detailHeader">
<tr>
<td class="detailHeaderName">getNextToken</td><td class="detailHeaderParens">()</td><td class="detailHeaderType">method</td><td class="detailHeaderRule">&nbsp;</td>
</tr>
</table>
<div class="detailBody">
<code>protected function getNextToken(consumeSource:Boolean = true):<a href="../parsing/Token.html">Token</a></code><span class="label">Parameters</span>
<table border="0" cellspacing="0" cellpadding="0">
<tr>
<td width="20px"></td><td><code><span class="label">consumeSource</span>:Boolean</code> (default = <code>true</code>)<code></code></td>
</tr>
</table>
<p></p>
<span class="label">Returns</span>
<table border="0" cellspacing="0" cellpadding="0">
<tr>
<td width="20"></td><td><code><a href="../parsing/Token.html">Token</a></code></td>
</tr>
</table>
</div>
<a name="lookAhead()"></a>
<table cellspacing="0" cellpadding="0" class="detailHeader">
<tr>
<td class="detailHeaderName">lookAhead</td><td class="detailHeaderParens">()</td><td class="detailHeaderType">method</td><td class="detailHeaderRule">&nbsp;</td>
</tr>
</table>
<div class="detailBody">
<code>public function lookAhead():<a href="../parsing/Token.html">Token</a></code>
<p></p>
<span class="label">Returns</span>
<table border="0" cellspacing="0" cellpadding="0">
<tr>
<td width="20"></td><td><code><a href="../parsing/Token.html">Token</a></code></td>
</tr>
</table>
</div>
<a name="nextToken()"></a>
<table cellspacing="0" cellpadding="0" class="detailHeader">
<tr>
<td class="detailHeaderName">nextToken</td><td class="detailHeaderParens">()</td><td class="detailHeaderType">method</td><td class="detailHeaderRule">&nbsp;</td>
</tr>
</table>
<div class="detailBody">
<code>public function nextToken():<a href="../parsing/Token.html">Token</a></code>
<p></p>
<span class="label">Returns</span>
<table border="0" cellspacing="0" cellpadding="0">
<tr>
<td width="20"></td><td><code><a href="../parsing/Token.html">Token</a></code></td>
</tr>
</table>
</div>
<a name="reset()"></a>
<table cellspacing="0" cellpadding="0" class="detailHeader">
<tr>
<td class="detailHeaderName">reset</td><td class="detailHeaderParens">()</td><td class="detailHeaderType">method</td><td class="detailHeaderRule">&nbsp;</td>
</tr>
</table>
<div class="detailBody">
<code>public function reset():void</code>
</div>
<a name="skipToken()"></a>
<table cellspacing="0" cellpadding="0" class="detailHeader">
<tr>
<td class="detailHeaderName">skipToken</td><td class="detailHeaderParens">()</td><td class="detailHeaderType">method</td><td class="detailHeaderRule">&nbsp;</td>
</tr>
</table>
<div class="detailBody">
<code>public function skipToken():void</code>
</div>
<br>
<br>
<hr>
<br>
<p></p>
<center class="copyright"> 
</center>
</div>
</body>
</html>
<!-- -->
